Reproducible Documentation of Analysis Study 1

Import of the Data

The data was assessed with the formr survey framework (Arslan, Walther, and Tata 2020). The raw data was imported via the following code.

library(tidyverse)
── Attaching packages ─────────────────────────────────────── tidyverse 1.3.2 ──
✔ ggplot2 3.4.0      ✔ purrr   0.3.5 
✔ tibble  3.1.8      ✔ dplyr   1.0.10
✔ tidyr   1.2.1      ✔ stringr 1.4.1 
✔ readr   2.1.3      ✔ forcats 0.5.2 
── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()
library(ggforce)
library(lme4)
Loading required package: Matrix

Attaching package: 'Matrix'

The following objects are masked from 'package:tidyr':

    expand, pack, unpack
library(BFpack)
Loading required package: bain
Registered S3 methods overwritten by 'BFpack':
  method               from
  get_estimates.lm     bain
  get_estimates.t_test bain
library(hrbrthemes)
library(patchwork)
library(brms)
Loading required package: Rcpp
Loading 'brms' package (version 2.18.0). Useful instructions
can be found by typing help('brms'). A more detailed introduction
to the package is available through vignette('brms_overview').

Attaching package: 'brms'

The following object is masked from 'package:lme4':

    ngrps

The following object is masked from 'package:stats':

    ar
library(viridis)
Loading required package: viridisLite
library(ggdist)

Attaching package: 'ggdist'

The following objects are masked from 'package:brms':

    dstudent_t, pstudent_t, qstudent_t, rstudent_t
library(tidybayes)

Attaching package: 'tidybayes'

The following objects are masked from 'package:brms':

    dstudent_t, pstudent_t, qstudent_t, rstudent_t
load("../data/teachers_study1_N40.RData")

set.seed(25051982)

Data Wrangling

# wrangle information on the plot type, ES, ...
plot_info <- study1 %>%
    pivot_longer(2:195, names_to = "variables", values_to = "values", 
                 values_transform = as.character) %>%
    dplyr::filter(str_detect(variables, "plot")) %>% 
    # we only need the rows with info on plots
    tidyr::separate(col = values, into = c("type", "axis", "effsize"), 
                    # separate the info into three columns
                    sep = "_", remove = F) %>%
    dplyr::mutate(plot = variables,       # rename variables for later join
                  type = paste(type, axis, sep = "_")) %>%
    dplyr::select(-variables, -axis)

# wrangle answers to items on each page
item_values <- study1 %>%
    dplyr::select(-c(topic:itemo)) %>%
    pivot_longer(2:169, names_to = "variables", values_to = "values", 
                 values_transform = as.character) %>%
    dplyr::mutate(variables = case_when(      # recode variable names that have
        variables == "sensi_6" ~ "sensi_06",  # accidentally been labeled
        variables == "acccl_6" ~ "acccl_06",  # without zero
        variables == "accu3_6" ~ "accu3_06",
        variables == "accov_6" ~ "accov_06",
        variables == "diffi_6" ~ "diffi_06",
        variables == "infor_6" ~ "infor_06",
        variables == "value_6" ~ "value_06",
        TRUE ~ variables 
    )) %>%
    dplyr::mutate(plot = paste0("plotx_", str_sub(variables, -2, -1)), 
                  # create variable for later join
                  variables = str_sub(variables, 1, -4)) %>%    
    # rename variable names to get a data set 
    # with one line per participant per page
    pivot_wider(id_cols = c(session, plot), names_from = "variables", 
                values_from = "values")


# join the two data sets
study1_w <- full_join(plot_info, item_values, 
                               by = c("session", "plot")) %>% 
    # by participant and page (plot)
    dplyr::select(-values) %>%
    dplyr::mutate(rating_cl = as.numeric(acccl), # some var need to be defined as
                  rating_u3 = as.numeric(accu3), # numeric again
                  rating_ov = as.numeric(accov),
                  diffi = as.numeric(diffi),
                  infor = as.numeric(infor),
                  value = as.numeric(value),
                  effsize = as.numeric(effsize),
                  effsize_cl = case_when( 
                  # there is no negative Cliff's Delta, so we have to compute 
                  # two transformations
                      effsize > 0 ~   (((2*pnorm(effsize/2))-1)/pnorm(effsize/2)),
                  # transform the actual effect size Cohen's d to Cliff's Delta
                      effsize < 0 ~ (- (((2*pnorm(abs(effsize)/2))-1)/pnorm(abs(effsize)/2))) 
                  # transform the actual effect size Cohen's d to Cliff's Delta 
                  # and make it negative as in the item
                  ),
                  effsize_u3 = 1-pnorm(effsize), # reverse so that it fits the direction of the U3 item
                  # transform the actual effect size Cohen's d to Cohen's U3
                  effsize_ov = 2 * pnorm(-abs(effsize) / 2), 
                  # transform the actual effect size Cohen's d to overlap
                  # actual difference of rating relative to depicted effectsize 
                  diff_cl = (rating_cl - effsize_cl)/2,
                  # actual difference of rating relative to depicted effectsize
                  diff_u3 = (rating_u3/100) - effsize_u3,
                  # actual difference of rating relative to depicted effectsize 
                  diff_ov = (rating_ov/100) - effsize_ov,
                  diffi_normed = ((diffi - 1)  / 3) - 1, # transform item to -1 to 1
                  infor_normed = ((infor - 1)  / 3) - 1, # transform item to -1 to 1
                  value_normed = ((value - 1)  / 3) - 1) %>%  # transform item to -1 to 1
    group_by(session) %>% 
    mutate(rating_ov_missconcept = median(rating_ov, na.rm = T) < 68.9,
           rating_u3_missconcept = median(rating_u3, na.rm = T) < 21.2) %>% 
    ungroup() %>% 
    mutate(rating_u3_filtered = ifelse(rating_u3_missconcept == T, NA, rating_u3),
           rating_ov_filtered = ifelse(rating_ov_missconcept == T, NA, rating_ov),
           diff_u3_filtered = (rating_u3_filtered/100) - effsize_u3,
           diff_ov_filtered = (rating_ov_filtered/100) - effsize_ov,
           sensi_binary = ifelse(is.na(sensi), # 1 if NOT "equal"
                                        NA,
                                        as.numeric(!grepl("equal", sensi))),
           sensi_ordinal = ordered(factor(substr(sensi, 55, 100)),
                                   levels = c("inferior",
                                              "equal",
                                              "superior")),
           sensi_binary_filtered = case_when(sensi_ordinal == "equal" ~ 0, 
                                              (sensi_ordinal == "inferior" & 
                                                  effsize < 0) | 
                                                  (sensi_ordinal == "superior" & 
                                                  effsize > 0) ~ as.numeric(NA),
                                              TRUE ~ 1),
           effsize_abs = abs(effsize))

# create a list of u3_misconceptualizers
u3_misconceptualizers <-
    study1_w %>% 
    filter(rating_u3_missconcept == T) %>% 
    pull(session) %>% 
    unique()

# create a list of ov_misconceptualizers
ov_misconceptualizers <-
    study1_w %>% 
    filter(rating_ov_missconcept == T) %>% 
    pull(session) %>% 
    unique()    

### wrangle time stamp data ####################################################
study1_w_timestamp <- 
    read_csv("../data/teachers_study1_N40_detailed.csv") %>% 
    # filter participants from study1_w only
    filter(session %in% study1_w$session) %>% 
    # we only need vars sensitivity or accuracy
    dplyr::filter(str_detect(item_name, "sensi|acccl|accu3|accov")) %>%  
    # create var with plot number
    mutate(plot = paste0("plotx_", str_sub(item_name, -2, -1)),
           # recode wrong item labelling
           plot = ifelse(plot == "plotx__6", "plotx_06", plot)) %>% 
    relocate(session, plot) %>% 
    # delete the page number in item name
    mutate(item_name = str_sub(item_name, 1, 5)) %>%  
    pivot_wider(id_cols = c(session, plot), names_from = item_name, 
                values_from = answered_relative) %>% 
    rowwise() %>%
    # what was the time of the first item to be clicked?
    mutate(effic = min(sensi, acccl, accu3, accov, na.rm=T)) %>%
    ungroup() %>% 
    dplyr::select(session, plot, effic, sensi, acccl, accu3, accov) %>% 
    left_join(., study1_w %>% 
                  select(session, plot, type), by=c("session", "plot")) %>% 
    # generate data set so that the six plots from the same type are ordered
    # one after the other (and not 1-24)
    group_by(session, type) %>% 
    arrange(plot) %>% 
    mutate(plotNrWithin = 1:n()) %>%
    ungroup() %>% 
    group_by(plotNrWithin, type) %>% 
    mutate(effic_10righttrunc = ifelse(effic > quantile(effic, .9), NA, effic),
           effic_05righttrunc = ifelse(effic > quantile(effic, .95), NA, effic),
           log_effic_05righttrunc =log(effic_05righttrunc),
           log_effic_10righttrunc = log(effic_10righttrunc),
           plotNrWithin0 = plotNrWithin -1,
           plotNrWithin_factor = as.factor(plotNrWithin)) %>% 
    ungroup()
Rows: 27336 Columns: 14
── Column specification ────────────────────────────────────────────────────────
Delimiter: ","
chr  (3): session, item_name, answer
dbl  (7): unit_session_id, item_id, shown_relative, answered_relative, displ...
dttm (4): created, saved, shown, answered

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

Accuracy

Distribution of the Dependent Variables

study1_w %>% 
    select(rating_cl, rating_u3, rating_ov) %>% 
    pivot_longer(
        c(rating_cl, rating_u3, rating_ov),
        names_to = "dependent_variable", 
        values_to = "rated_effectsize"
        ) %>% 
    ggplot(., aes(rated_effectsize)) +
    geom_histogram() +
    facet_wrap(~dependent_variable, scales = "free_x") +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white"))
`stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
Warning: Removed 1440 rows containing non-finite values (`stat_bin()`).

Somewhat disturbing is the first mode in rating_ov. Maybe some users confused overlap and non-overlap? Another artifact seems to be the first mode in rating_u3.

Are there Constant Misconceptions per Persons?

ggplot(study1_w %>% 
    select(rating_cl, rating_u3, rating_ov, effsize, effsize_cl, session) %>% 
    pivot_longer(
        c(rating_cl, rating_u3, rating_ov),
        names_to = "operationalization", 
        values_to = "judged_effectsize"
        ),
    aes(judged_effectsize, as.numeric(as.factor(session)),
        color = session)
        ) +
    geom_jitter(height = 0) +
    facet_wrap(~ operationalization, scales = "free_x") +
    theme_modern_rc() +
    theme(legend.position = "none",
          strip.text = element_text(color = "white"))
Warning: Removed 1440 rows containing missing values (`geom_point()`).

Association of Ratings and Actual Effect Size

study1_w %>% 
    ggplot(., aes(effsize, rating_cl)) +
    geom_jitter() +
    stat_smooth() +
    theme_modern_rc()
`geom_smooth()` using method = 'loess' and formula = 'y ~ x'
Warning: Removed 480 rows containing non-finite values (`stat_smooth()`).
Warning: Removed 480 rows containing missing values (`geom_point()`).

study1_w %>% 
    ggplot(., aes(effsize, rating_u3, color = rating_u3_missconcept)) +
    geom_jitter() +
    stat_smooth() +
    theme_modern_rc()
`geom_smooth()` using method = 'loess' and formula = 'y ~ x'
Warning: Removed 480 rows containing non-finite values (`stat_smooth()`).
Removed 480 rows containing missing values (`geom_point()`).

study1_w %>% 
    ggplot(., aes(abs(effsize), rating_ov, color = rating_ov_missconcept)) +
    geom_jitter() +
    stat_smooth(method = "lm") +
    theme_modern_rc()
`geom_smooth()` using formula = 'y ~ x'
Warning: Removed 480 rows containing non-finite values (`stat_smooth()`).
Removed 480 rows containing missing values (`geom_point()`).

The correlations underpin the interpretation of the misconceptions. Therefore, we will look at the intercorrelations of the dependent variables.

Compute Kendall’s \(\tau\) for rating_cl, rating_u3 and rating_ov

study1_w %>% 
    group_by(type) %>% 
    do(tau_cl = unlist(cor(.$effsize, .$rating_cl, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_cl)
# A tibble: 4 × 2
  type                tau_cl
  <chr>                <dbl>
1 gardneraltman_xaxis  0.373
2 halfeye_xaxis        0.548
3 halfeye_yaxis        0.484
4 raincloud_yaxis      0.415
study1_w %>% 
    group_by(rating_ov_missconcept, type) %>% 
    do(tau_ov = unlist(cor(abs(.$effsize), .$rating_ov, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_ov)
# A tibble: 8 × 3
  rating_ov_missconcept type                 tau_ov
  <lgl>                 <chr>                 <dbl>
1 FALSE                 gardneraltman_xaxis -0.355 
2 FALSE                 halfeye_xaxis       -0.384 
3 FALSE                 halfeye_yaxis       -0.478 
4 FALSE                 raincloud_yaxis     -0.523 
5 TRUE                  gardneraltman_xaxis  0.0236
6 TRUE                  halfeye_xaxis        0.119 
7 TRUE                  halfeye_yaxis        0.0205
8 TRUE                  raincloud_yaxis     -0.220 
study1_w %>% 
    group_by(rating_u3_missconcept, type) %>% 
    do(tau_u3 = cor(.$effsize, .$rating_u3, method = "kendall", 
                           use = "pairwise.complete")) %>% 
    unnest(tau_u3)
# A tibble: 8 × 3
  rating_u3_missconcept type                 tau_u3
  <lgl>                 <chr>                 <dbl>
1 FALSE                 gardneraltman_xaxis -0.298 
2 FALSE                 halfeye_xaxis       -0.471 
3 FALSE                 halfeye_yaxis       -0.263 
4 FALSE                 raincloud_yaxis     -0.321 
5 TRUE                  gardneraltman_xaxis  0.119 
6 TRUE                  halfeye_xaxis       -0.153 
7 TRUE                  halfeye_yaxis       -0.108 
8 TRUE                  raincloud_yaxis     -0.0971

Associations of the Dependent Variables

ggplot(study1_w, 
       aes(rating_cl, rating_u3)) +
    geom_jitter() +
    theme_modern_rc() +
ggplot(study1_w, 
       aes(rating_cl, rating_ov)) +
    geom_jitter() +
    theme_modern_rc() +
ggplot(study1_w, 
       aes(rating_ov, rating_u3)) +
    geom_jitter() +
    theme_modern_rc()
Warning: Removed 480 rows containing missing values (`geom_point()`).
Removed 480 rows containing missing values (`geom_point()`).
Removed 480 rows containing missing values (`geom_point()`).

Global Under- or Overestimation

study1_w %>% 
    select(diff_cl, diff_u3_filtered, diff_ov_filtered, session, type) %>% 
    gather(dependent_variable, difference_to_true_effsize, 
           diff_cl, diff_u3_filtered, diff_ov_filtered) %>% 
    ggplot(., aes(type, difference_to_true_effsize)) +
    geom_boxplot(alpha = .3) +
    facet_wrap(~dependent_variable, scales = "free_y") +
    geom_jitter(aes(color = type)) +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white"),
          axis.text.x = element_blank())
Warning: Removed 1812 rows containing non-finite values (`stat_boxplot()`).
Warning: Removed 1812 rows containing missing values (`geom_point()`).

Research Question 1

Intraclasscorrelations for Accuracy, Perceived Difficulty, Informativity and Value

icc_mod_rating_cl <- lmer(rating_cl ~  1 + (1|type), 
                          data = study1_w )
boundary (singular) fit: see help('isSingular')
icc_mod_rating_u3 <- lmer(rating_u3 ~  1 + (1|type), 
                          data = study1_w %>% 
                              filter(rating_u3_missconcept == F))
icc_mod_rating_ov <- lmer(rating_ov ~  1 + (1|type), 
                          data = study1_w %>% 
                              filter(rating_ov_missconcept == F))
icc_mod_diffi <- lmer(diffi ~  1 + (1|type), 
                      data = study1_w)
icc_mod_infor <- lmer(infor ~  1 + (1|type), 
                      data = study1_w )
icc_mod_value <- lmer(value ~  1 + (1|type), 
                      data = study1_w )

sjPlot::tab_model(
    icc_mod_rating_cl,
    icc_mod_rating_u3,
    icc_mod_rating_ov,
    icc_mod_diffi,
    icc_mod_infor,
    icc_mod_value
)
  rating_cl rating_u3 rating_ov diffi infor value
Predictors Estimates CI p Estimates CI p Estimates CI p Estimates CI p Estimates CI p Estimates CI p
(Intercept) 0.01 -0.02 – 0.05 0.391 43.08 39.89 – 46.27 <0.001 82.60 80.85 – 84.34 <0.001 4.05 3.63 – 4.47 <0.001 4.11 3.86 – 4.36 <0.001 4.13 3.87 – 4.39 <0.001
Random Effects
σ2 0.15 317.10 282.66 2.79 2.07 2.17
τ00 0.00 type 4.61 type 0.10 type 0.17 type 0.06 type 0.06 type
ICC   0.01 0.00 0.06 0.03 0.03
N 4 type 4 type 4 type 4 type 4 type 4 type
Observations 480 216 372 960 960 960
Marginal R2 / Conditional R2 0.000 / NA 0.000 / 0.014 0.000 / 0.000 0.000 / 0.058 0.000 / 0.026 0.000 / 0.028
BF(icc_mod_rating_cl, hypothesis = "icc = 0; icc > 0")
First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 
Call:
BF.lmerMod(x = icc_mod_rating_cl, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.375
H2               0.006
H3               0.619

Evidence matrix (Bayes factors):
      H1     H2    H3
H1 1.000 59.126 0.605
H2 0.017  1.000 0.010
H3 1.652 97.673 1.000

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement
BF(icc_mod_rating_u3, hypothesis = "icc = 0; icc > 0")
First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 
Call:
BF.lmerMod(x = icc_mod_rating_u3, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.657
H2               0.108
H3               0.235

Evidence matrix (Bayes factors):
      H1    H2    H3
H1 1.000 6.075 2.793
H2 0.165 1.000 0.460
H3 0.358 2.175 1.000

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement
BF(icc_mod_rating_ov, hypothesis = "icc = 0; icc > 0")
First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 
Call:
BF.lmerMod(x = icc_mod_rating_ov, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.605
H2               0.027
H3               0.368

Evidence matrix (Bayes factors):
      H1     H2    H3
H1 1.000 22.432 1.648
H2 0.045  1.000 0.073
H3 0.607 13.615 1.000

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement
BF(icc_mod_diffi, hypothesis = "icc = 0; icc > 0")
First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 
Call:
BF.lmerMod(x = icc_mod_diffi, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1                   0
H2                   1
H3                   0

Evidence matrix (Bayes factors):
        H1 H2  H3
H1       1  0 Inf
H2 7895605  1 Inf
H3       0  0   1

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement
BF(icc_mod_infor, hypothesis = "icc = 0; icc > 0")
First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 
Call:
BF.lmerMod(x = icc_mod_infor, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.012
H2               0.988
H3               0.000

Evidence matrix (Bayes factors):
       H1    H2  H3
H1  1.000 0.012 Inf
H2 85.355 1.000 Inf
H3  0.000 0.000   1

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement
BF(icc_mod_value, hypothesis = "icc = 0; icc > 0")
First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 
Call:
BF.lmerMod(x = icc_mod_value, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.006
H2               0.994
H3               0.000

Evidence matrix (Bayes factors):
        H1    H2  H3
H1   1.000 0.006 Inf
H2 174.388 1.000 Inf
H3   0.000 0.000   1

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement

Random Intercept Models With and Without Visualization Type

# Rating Cliffs Delta
mod0_rating_cl <- brm(rating_cl ~ + (1|session), 
                          data = study1_w,
                          iter = 20000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4)
Warning: Rows containing NAs were excluded from the model.
Compiling Stan program...
Trying to compile a simple C file
Start sampling
mod1_rating_cl <- brm(rating_cl ~ effsize + (1|session), 
                          data = study1_w,
                          iter = 20000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4)
Warning: Rows containing NAs were excluded from the model.
Compiling Stan program...
Trying to compile a simple C file
Start sampling
mod2_rating_cl <- brm(rating_cl ~ type + effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Warning: Rows containing NAs were excluded from the model.
Compiling Stan program...
Trying to compile a simple C file
Start sampling
bayes_factor(mod1_rating_cl, mod0_rating_cl)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod1_rating_cl over mod0_rating_cl: 693132216972923326062318468464640.00000
bayes_factor(mod2_rating_cl, mod1_rating_cl)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Estimated Bayes factor in favor of mod2_rating_cl over mod1_rating_cl: 0.00113
# Rating Cohen's U3
mod0_rating_u3 <- brm(rating_u3 ~ + (1|session), 
                          data = study1_w%>% 
                              filter(rating_u3_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Warning: Rows containing NAs were excluded from the model.
Compiling Stan program...
Trying to compile a simple C file
Start sampling
mod1_rating_u3 <- brm(rating_u3 ~ effsize + (1|session), 
                          data = study1_w%>% 
                              filter(rating_u3_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Warning: Rows containing NAs were excluded from the model.
Compiling Stan program...
Trying to compile a simple C file
Start sampling
mod2_rating_u3 <- brm(rating_u3 ~ type + effsize + (1|session), 
                          data = study1_w %>% 
                              filter(rating_u3_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Warning: Rows containing NAs were excluded from the model.
Compiling Stan program...
Trying to compile a simple C file
Start sampling
bayes_factor(mod1_rating_u3, mod0_rating_u3)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Estimated Bayes factor in favor of mod1_rating_u3 over mod0_rating_u3: 1078905234.49353
bayes_factor(mod2_rating_u3, mod1_rating_u3)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Estimated Bayes factor in favor of mod2_rating_u3 over mod1_rating_u3: 2459.95439
# Rating Overlap
mod0_rating_ov <- brm(rating_ov ~ + (1|session), 
                          data = study1_w%>% 
                              filter(rating_ov_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Warning: Rows containing NAs were excluded from the model.
Compiling Stan program...
Trying to compile a simple C file
Start sampling
mod1_rating_ov <- brm(rating_ov ~ effsize + (1|session), 
                          data = study1_w%>% 
                              filter(rating_ov_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Warning: Rows containing NAs were excluded from the model.
Compiling Stan program...
Trying to compile a simple C file
Start sampling
mod2_rating_ov <- brm(rating_ov ~ type + effsize + (1|session), 
                          data = study1_w %>% 
                              filter(rating_ov_missconcept == F),
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Warning: Rows containing NAs were excluded from the model.
Compiling Stan program...
Trying to compile a simple C file
Start sampling
bayes_factor(mod1_rating_ov, mod0_rating_ov)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Estimated Bayes factor in favor of mod1_rating_ov over mod0_rating_ov: 4.96682
bayes_factor(mod2_rating_ov, mod1_rating_ov)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod2_rating_ov over mod1_rating_ov: 522.76670
# Perceived Difficulty
mod0_diffi <- brm(diffi ~ + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Compiling Stan program...
Trying to compile a simple C file
Start sampling
mod1_diffi <- brm(diffi ~ effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Compiling Stan program...
Trying to compile a simple C file
Start sampling
mod2_diffi <- brm(diffi ~ type + effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Compiling Stan program...
Trying to compile a simple C file
Start sampling
bayes_factor(mod1_diffi, mod0_diffi)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod1_diffi over mod0_diffi: 0.39640
bayes_factor(mod2_diffi, mod1_diffi)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod2_diffi over mod1_diffi: 4800845358254093.00000
# Perceived Informativity
mod0_infor <- brm(infor ~ + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Compiling Stan program...
Trying to compile a simple C file
Start sampling
mod1_infor <- brm(infor ~ effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Compiling Stan program...
Trying to compile a simple C file
Start sampling
mod2_infor <- brm(infor ~ type + effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Compiling Stan program...
Trying to compile a simple C file
Start sampling
bayes_factor(mod1_infor, mod0_infor)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod1_infor over mod0_infor: 0.26039
bayes_factor(mod2_infor, mod1_infor)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Estimated Bayes factor in favor of mod2_infor over mod1_infor: 742781.83099
# Perceived Value 
mod0_value <- brm(value ~ + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Compiling Stan program...
Trying to compile a simple C file
Start sampling
mod1_value <- brm(value ~ effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Compiling Stan program...
Trying to compile a simple C file
Start sampling
mod2_value <- brm(value ~ type + effsize + (1|session), 
                          data = study1_w,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE),
                          cores = 4
                          )
Compiling Stan program...
Trying to compile a simple C file
Start sampling
bayes_factor(mod1_value, mod0_value)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of mod1_value over mod0_value: 0.20633
bayes_factor(mod2_value, mod1_value)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Estimated Bayes factor in favor of mod2_value over mod1_value: 84181532.57805
sjPlot::tab_model(
    mod0_rating_cl, mod0_rating_u3, mod0_rating_ov,
    mod0_diffi, mod0_infor, mod0_value,
    mod1_rating_cl, mod1_rating_u3, mod1_rating_ov,
    mod1_diffi, mod1_infor, mod1_value,
    mod2_rating_cl, mod2_rating_u3, mod2_rating_ov,
    mod2_diffi, mod2_infor, mod2_value
    )
  rating_cl rating_u3 rating_ov diffi infor value rating_cl rating_u3 rating_ov diffi infor value rating_cl rating_u3 rating_ov diffi infor value
Predictors Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%)
Intercept 0.01 -0.02 – 0.05 43.10 38.59 – 47.73 82.66 80.13 – 85.23 4.04 3.68 – 4.42 4.11 3.80 – 4.42 4.13 3.79 – 4.45 0.01 -0.02 – 0.04 43.32 38.67 – 48.08 82.68 80.23 – 85.21 4.04 3.68 – 4.41 4.11 3.81 – 4.41 4.12 3.78 – 4.47 0.02 -0.05 – 0.08 42.65 36.83 – 48.45 82.35 78.31 – 86.19 3.59 3.20 – 3.99 3.81 3.48 – 4.13 3.79 3.43 – 4.16
effsize 0.37 0.32 – 0.42 -11.82 -15.36 – -8.26 -1.03 -4.01 – 1.95 -0.09 -0.24 – 0.06 -0.06 -0.19 – 0.07 -0.05 -0.17 – 0.08 0.37 0.32 – 0.42 -11.64 -15.24 – -8.09 -1.07 -4.04 – 1.96 -0.09 -0.23 – 0.05 -0.06 -0.19 – 0.07 -0.05 -0.17 – 0.07
typehalfeye_xaxis -0.02 -0.10 – 0.06 0.58 -4.93 – 6.13 2.48 -2.19 – 7.14 0.65 0.42 – 0.87 0.39 0.19 – 0.59 0.51 0.32 – 0.70
typehalfeye_yaxis 0.01 -0.08 – 0.09 4.15 -1.60 – 9.78 -1.01 -5.84 – 3.84 0.97 0.74 – 1.19 0.60 0.40 – 0.80 0.58 0.39 – 0.77
typeraincloud_yaxis 0.01 -0.08 – 0.09 -1.87 -7.60 – 3.81 -0.14 -4.90 – 4.67 0.24 0.02 – 0.47 0.24 0.04 – 0.45 0.23 0.04 – 0.42
Random Effects
σ2 0.15 259.01 254.58 1.72 1.32 1.17 0.11 213.63 255.25 1.72 1.32 1.17 0.11 212.62 256.13 1.58 1.27 1.11
τ00 0.00 session 75.49 session 30.02 session 1.30 session 0.86 session 1.15 session 0.00 session 82.97 session 29.99 session 1.31 session 0.87 session 1.15 session 0.00 session 80.38 session 28.66 session 1.32 session 0.87 session 1.13 session
ICC 0.01 0.23 0.11 0.43 0.39 0.50 0.02 0.28 0.11 0.43 0.40 0.50 0.02 0.27 0.10 0.46 0.40 0.50
N 40 session 18 session 31 session 40 session 40 session 40 session 40 session 18 session 31 session 40 session 40 session 40 session 40 session 18 session 31 session 40 session 40 session 40 session
Observations 480 216 372 960 960 960 480 216 372 960 960 960 480 216 372 960 960 960
Marginal R2 / Conditional R2 0.000 / 0.006 0.000 / 0.198 0.000 / 0.099 0.000 / 0.413 0.000 / 0.376 0.000 / 0.476 0.278 / 0.291 0.134 / 0.344 0.002 / 0.102 0.001 / 0.415 0.001 / 0.377 0.000 / 0.476 0.282 / 0.294 0.154 / 0.358 0.015 / 0.111 0.050 / 0.464 0.025 / 0.401 0.026 / 0.501

Perceived Difficulty

Graphichal Overview

study1_w %>% 
    ggplot(aes(type, diffi)) +
    geom_jitter() +
    stat_summary(fun.data = mean_sdl,
                 fun.args = list(mult = 1),
                 color = "white") +
    theme_modern_rc() +
    labs(title = "Difficulty",
         subtitle = "per Plot Type",
         caption = "Means ± 1*SD")

Perceived Informativity

Graphical Overview

study1_w %>% 
    ggplot(aes(type, infor)) +
    geom_jitter() +
    stat_summary(fun.data = mean_sdl,
                 fun.args = list(mult = 1),
                 color = "white") +
    theme_modern_rc() +
    labs(title = "Informativity",
         subtitle = "per Plot Type",
         caption = "Means ± 1*SD")

Perceived Value

Graphical Overview

study1_w %>% 
    ggplot(aes(type, value)) +
    geom_jitter() +
    stat_summary(fun.data = mean_sdl,
                 fun.args = list(mult = 1),
                 color = "white") +
    theme_modern_rc() +
    labs(title = "Perceived Value",
         subtitle = "per Plot Type",
         caption = "Means ± 1*SD")

Sensitivity

ggplot(study1_w %>% 
           select(sensi_ordinal, type) %>% 
           na.omit(), aes(sensi_ordinal, 
                          color = sensi_ordinal,
                          fill = sensi_ordinal)) +
    facet_wrap(~ type) +
    geom_bar() +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white")) +
    ggtitle("Sensitivity", "all participants")

ggplot(study1_w %>% 
           select(sensi_ordinal, type, session) %>% 
           na.omit() %>% 
           filter(!session %in% ov_misconceptualizers & 
                      !session %in% u3_misconceptualizers), aes(sensi_ordinal, 
                          color = sensi_ordinal,
                          fill = sensi_ordinal)) +
    facet_wrap(~ type) +
    geom_bar() +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white")) +
    ggtitle("Sensitivity", "without missconceptualizers")

ggplot(study1_w %>% 
           select(sensi_binary_filtered, type, session),
       aes(sensi_binary_filtered, 
           color = sensi_binary_filtered,
           fill = sensi_binary_filtered)) +
    facet_wrap(~ type) +
    geom_bar() +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white")) +
    ggtitle("Binary Sensitivity", "without wrong decisions")

Fitting a series of logistic regressions with non-informative priors

logreg_mod0 <- brm(
    sensi_binary_filtered ~ +(1 | session),
    family = bernoulli(link = "logit"),
    data = study1_w,
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4
    
)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
logreg_mod1 <-
    brm(
        sensi_binary_filtered ~ effsize_abs + (1 | session),
        family = bernoulli(link = "logit"),
        data = study1_w,
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4
        
    )
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
logreg_mod2 <-
    brm(
        sensi_binary_filtered ~ effsize_abs + type + (1 | session),
        family = bernoulli(link = "logit"),
        data = study1_w,
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4
        
    )
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
bayes_factor(logreg_mod1, logreg_mod0)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Estimated Bayes factor in favor of logreg_mod1 over logreg_mod0: 979646073651857.50000
bayes_factor(logreg_mod2, logreg_mod1)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Estimated Bayes factor in favor of logreg_mod2 over logreg_mod1: 2.14972
sjPlot::tab_model(logreg_mod0, logreg_mod1, logreg_mod2, show.icc = TRUE)
  sensi_binary_filtered sensi_binary_filtered sensi_binary_filtered
Predictors Odds Ratios CI (95%) Odds Ratios CI (95%) Odds Ratios CI (95%)
Intercept 7.75 5.20 – 12.94 1.67 0.96 – 3.10 1.45 0.72 – 2.93
effsize_abs 36.73 14.94 – 91.86 37.60 15.08 – 99.41
typehalfeye_xaxis 1.56 0.89 – 2.84
typehalfeye_yaxis 1.36 0.78 – 2.42
typeraincloud_yaxis 0.95 0.55 – 1.63
Random Effects
σ2 3.29 3.29 3.29
τ00 1.32 session 1.61 session 1.65 session
ICC 0.29 0.33 0.33
N 40 session 40 session 40 session
Observations 933 933 933
Marginal R2 / Conditional R2 0.000 / 0.115 0.051 / 0.200 0.056 / 0.208
pp_check(logreg_mod2) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))
Using 10 posterior draws for ppc type 'dens_overlay' by default.
Scale for colour is already present.
Adding another scale for colour, which will replace the existing scale.

Efficiency

Visualisation

Raw data

ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), effic)) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    coord_cartesian(ylim = c(0,100000)) + 
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))

5% Percent Truncated

ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), effic_05righttrunc)) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    coord_cartesian(ylim = c(0,85000)) + 
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="5% Truncated Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))
Warning: Removed 48 rows containing non-finite values (`stat_boxplot()`).
Warning: Removed 48 rows containing non-finite values (`stat_sina()`).

10% Percent Truncated

ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), effic_10righttrunc)) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    coord_cartesian(ylim = c(0,85000)) + 
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="10% Truncated Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))
Warning: Removed 96 rows containing non-finite values (`stat_boxplot()`).
Warning: Removed 96 rows containing non-finite values (`stat_sina()`).

log() Transformed

ggplot(study1_w_timestamp, 
       aes(as.factor(plotNrWithin), 
           log(effic_05righttrunc))) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="log Transformed Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))
Warning: Removed 48 rows containing non-finite values (`stat_boxplot()`).
Warning: Removed 48 rows containing non-finite values (`stat_sina()`).

Modeling with {brms}

Efficiency for First Plot

study1_w_timestamp_effsize <- study1_w_timestamp %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")

plot01_mod0 <-
    brm(log_effic_05righttrunc ~ 1 + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin == 1),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4,
        iter = 6000)

pp_check(plot01_mod0) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

plot01_mod1 <-
    brm(log_effic_05righttrunc~ 1 + effsize + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin == 1),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4,
        iter = 6000)

pp_check(plot01_mod1) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

plot01_mod2 <-
    brm(log_effic_05righttrunc ~ 1 + effsize + type + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin == 1),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4,
        iter = 6000)

pp_check(plot01_mod2) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

sjPlot::tab_model(plot01_mod0, plot01_mod1, plot01_mod2)
bayes_factor(plot01_mod1, plot01_mod0)
bayes_factor(plot01_mod2, plot01_mod1)

Efficiency for Last Three Plots

plot0306_mod0 <-
    brm(log_effic_05righttrunc ~ 1 + plotNrWithin_factor + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin >= 4),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4,
        iter = 6000)

pp_check(plot0306_mod0) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

plot0306_mod1 <-
    brm(log_effic_05righttrunc ~ 1 + plotNrWithin_factor + effsize +
            (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin >= 4),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4)

pp_check(plot0306_mod1) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

plot0306_mod2 <-
    brm(log_effic_05righttrunc ~ 1 + plotNrWithin_factor + effsize +
            type + (1 | session),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin >= 4),
        save_pars = save_pars(all = TRUE),
        silent = 2,
        refresh = 0,
        cores = 4)

pp_check(plot0306_mod2) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

sjPlot::tab_model(plot0306_mod0, plot0306_mod1, plot0306_mod2)
  log_effic_05righttrunc log_effic_05righttrunc log_effic_05righttrunc
Predictors Estimates CI (95%) Estimates CI (95%) Estimates CI (95%)
Intercept 9.48 9.34 – 9.62 9.48 9.35 – 9.60 9.58 9.45 – 9.71
plotNrWithin_factor:
plotNrWithin_factor5
-0.01 -0.03 – 0.02 -0.01 -0.03 – 0.02 -0.01 -0.03 – 0.01
plotNrWithin_factor:
plotNrWithin_factor6
-0.19 -0.21 – -0.17 -0.19 -0.21 – -0.17 -0.19 -0.21 – -0.17
effsize -0.00 -0.02 – 0.02 0.00 -0.02 – 0.02
typehalfeye_xaxis -0.13 -0.16 – -0.11
typehalfeye_yaxis -0.21 -0.23 – -0.18
typeraincloud_yaxis -0.06 -0.09 – -0.04
Random Effects
σ2 0.24 0.24 0.24
τ00 0.17 session 0.17 session 0.16 session
ICC 0.41 0.41 0.41
N 40 session 40 session 40 session
Observations 10944 10944 10944
Marginal R2 / Conditional R2 0.019 / 0.406 0.019 / 0.406 0.034 / 0.421
bayes_factor(plot0306_mod1, plot0306_mod0)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 7
Iteration: 8
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 7
Estimated Bayes factor in favor of plot0306_mod1 over plot0306_mod0: 0.02010
bayes_factor(plot0306_mod2, plot0306_mod1)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 7
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Estimated Bayes factor in favor of plot0306_mod2 over plot0306_mod1: 18618875449292113148499011020035359633188591249536319488.00000

Research Question 2

Contrast Effects

Graphical Overview

# preparing multilevel model with brms for graphical overview

# rating u3
mod_plot_ratingu3 <- brm(
    scale(rating_u3) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4,
    iter = 20000
    
) 
Warning: Rows containing NAs were excluded from the model.
Compiling Stan program...
Trying to compile a simple C file
Start sampling
Warning: There were 4 divergent transitions after warmup. See
https://mc-stan.org/misc/warnings.html#divergent-transitions-after-warmup
to find out why this is a problem and how to eliminate them.
Warning: Examine the pairs() plot to diagnose sampling problems
plot_ratingu3 <- mod_plot_ratingu3 %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean
             #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
    stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_text(hjust = 0)) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Accuracy in U3 Metrics')
Warning: `gather_()` was deprecated in tidyr 1.2.0.
ℹ Please use `gather()` instead.
ℹ The deprecated feature was likely used in the tidybayes package.
  Please report the issue at <]8;;https://github.com/mjskay/tidybayes/issues/newhttps://github.com/mjskay/tidybayes/issues/new]8;;>.
plot_ratingu3
Warning: Using the `size` aesthietic with geom_segment was deprecated in ggplot2 3.4.0.
ℹ Please use the `linewidth` aesthetic instead.

# rating ov
mod_plot_ratingov <- brm (
    scale(rating_ov) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4,
    iter = 10000
    
)
Warning: Rows containing NAs were excluded from the model.
Compiling Stan program...
Trying to compile a simple C file
Start sampling
plot_ratingov <-mod_plot_ratingov %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean
             #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
    stat_halfeye() + 
    theme_ipsum_rc()+
    theme(axis.title.x = element_blank(),
          axis.text.y = element_blank(), 
          axis.title.y = element_blank()) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Accuracy in Overlap Metrics')
plot_ratingov

# difficulty
mod_plot_diffi <- brm (
    scale(diffi) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4
    
)
Compiling Stan program...
Trying to compile a simple C file
Start sampling
plot_diffi <- mod_plot_diffi %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean
             #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
    stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_blank(), 
          axis.title.y = element_blank()) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Perceived Task Difficulty')
plot_diffi

# informativity
mod_plot_infor <- brm (
    scale(infor) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4,
    iter = 10000
    
)
Compiling Stan program...
Trying to compile a simple C file
Start sampling
plot_infor <- mod_plot_infor %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
  stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_text(hjust = 0)) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Perceived Informativity')
plot_infor

# value 

mod_plot_value <- brm (
    scale(value) ~ (1 | type),
    data = study1_w,
    control = list(adapt_delta = .999),
    cores = 4
    
)
Compiling Stan program...
Trying to compile a simple C file
Start sampling
plot_value <- mod_plot_value %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
  stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_blank(), 
          axis.title.y = element_blank()) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Perceived Value')
plot_value

# efficiency for the first plot

mod_plot_efficiency1 <-
    brm(
        scale(effic) ~ (1 | type),
        data = study1_w_timestamp_effsize %>%
            filter(plotNrWithin == 1),
        control = list(adapt_delta = .999),
        cores = 4
        
    )
Compiling Stan program...
Trying to compile a simple C file
Start sampling
Warning: There were 3 transitions after warmup that exceeded the maximum treedepth. Increase max_treedepth above 10. See
https://mc-stan.org/misc/warnings.html#maximum-treedepth-exceeded
Warning: Examine the pairs() plot to diagnose sampling problems
plot_efficiency1 <- mod_plot_efficiency1 %>%
  spread_draws(b_Intercept, r_type[type,]) %>%
 # median_qi(type_mean = b_Intercept + r_type, .width = c(.95, .66)) %>%
  mutate(type_mean = b_Intercept + r_type) %>%
  ggplot(aes(y = type, x = type_mean #xmin = .lower, xmax = .upper
             )) +
#  geom_pointinterval() +
  stat_halfeye() +
    theme_ipsum_rc() +
    theme(axis.title.x = element_blank(),
          axis.text.y = element_blank(), 
          axis.title.y = element_blank()) +
    #scale_x_continuous(limits=c(-0.6, 0.6)) +
    ggtitle('Efficiency for the First Plot')
plot_efficiency1

plot_contrasteffects <- plot_ratingu3 + plot_ratingov + plot_diffi + plot_infor + plot_value + plot_efficiency1 + 
    plot_layout(ncol=3)

ggsave("plot_contrasteffects.png", 
       plot_contrasteffects,
       width = 25*.55,
       height = 16*.55)

Model Contrast Effects with {brms}

# create dummy variables to compare visualization types for those measures that showed evidence for difference (accuracy U3, accuracy overlap, perceived task difficulty, perceived informativity, perceived value, efficiency)

## Gardner Altman Plot vs. Halfeye xaxis
gardneraltman_vs_halfeye_x <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_halfeye_x = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "halfeye_xaxis") %>%
    ungroup()

gardneraltman_vs_halfeye_x_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_halfeye_x = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "halfeye_xaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")
mod_accu3_gardneraltman_halfeye_x <- 
    brm(scale(rating_u3) ~ dummy_gardneraltman_halfeye_x + effsize + (1|session), 
                          data = gardneraltman_vs_halfeye_x%>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_accov_gardneraltman_halfeye_x <- 
    brm(scale(rating_ov) ~ dummy_gardneraltman_halfeye_x + effsize + (1|session), 
                          data = gardneraltman_vs_halfeye_x%>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_diffi_gardneraltman_halfeye_x <- 
    brm(scale(diffi) ~ dummy_gardneraltman_halfeye_x + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Trying to compile a simple C file
mod_infor_gardneraltman_halfeye_x <- 
    brm(scale(infor) ~ dummy_gardneraltman_halfeye_x + effsize
        + (1|session), 
                          data = gardneraltman_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Trying to compile a simple C file
mod_value_gardneraltman_halfeye_x <- 
    brm(scale(value) ~ dummy_gardneraltman_halfeye_x + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE))
Compiling Stan program...
Trying to compile a simple C file
Start sampling
Warning: Bulk Effective Samples Size (ESS) is too low, indicating posterior means and medians may be unreliable.
Running the chains for more iterations may help. See
https://mc-stan.org/misc/warnings.html#bulk-ess
mod_effic01_gardneraltman_halfeye_x <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_halfeye_x + 
            effsize + (1|session),
    data = gardneraltman_vs_halfeye_x_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_effic0306_gardneraltman_halfeye_x <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_halfeye_x +
            effsize + (1|session),
    data = gardneraltman_vs_halfeye_x_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
## Gardner Altman Plot vs. Halfeye yaxis
gardneraltman_vs_halfeye_y <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_halfeye_y = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "halfeye_yaxis") %>%
    ungroup()
gardneraltman_vs_halfeye_y_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_halfeye_y = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "halfeye_yaxis") %>%
    ungroup()  %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")


mod_accu3_gardneraltman_halfeye_y <- 
    brm(scale(rating_u3) ~ dummy_gardneraltman_halfeye_y + effsize + (1|session), 
                          data = gardneraltman_vs_halfeye_y%>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_accov_gardneraltman_halfeye_y <- 
    brm(scale(rating_ov) ~ dummy_gardneraltman_halfeye_y + effsize + (1|session), 
                          data = gardneraltman_vs_halfeye_y%>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_diffi_gardneraltman_halfeye_y <- 
    brm(scale(diffi) ~ dummy_gardneraltman_halfeye_y + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Trying to compile a simple C file
mod_infor_gardneraltman_halfeye_y <- 
    brm(scale(infor) ~ dummy_gardneraltman_halfeye_y + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Trying to compile a simple C file
mod_value_gardneraltman_halfeye_y <- 
    brm(scale(value) ~ dummy_gardneraltman_halfeye_y + effsize 
        + (1|session), 
                          data = gardneraltman_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Trying to compile a simple C file
mod_effic01_gardneraltman_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_halfeye_y +
            effsize + (1|session),
    data = gardneraltman_vs_halfeye_y_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
Warning: Bulk Effective Samples Size (ESS) is too low, indicating posterior means and medians may be unreliable.
Running the chains for more iterations may help. See
https://mc-stan.org/misc/warnings.html#bulk-ess
Warning: Tail Effective Samples Size (ESS) is too low, indicating posterior variances and tail quantiles may be unreliable.
Running the chains for more iterations may help. See
https://mc-stan.org/misc/warnings.html#tail-ess
mod_effic0306_gardneraltman_halfeye_y <- 
    brm(log_effic_05righttrunc ~ dummy_gardneraltman_halfeye_y + 
            effsize + (1|session),
    data = gardneraltman_vs_halfeye_y_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
## Raincloud Plot vs. Halfeye x axis
raincloud_vs_halfeye_x <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_raincloud_halfeye_x = 
               case_when(type == "raincloud_yaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "halfeye_xaxis" | type == "raincloud_yaxis") %>%
    ungroup()

raincloud_vs_halfeye_x_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_raincloud_halfeye_x = 
               case_when(type == "raincloud_yaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "raincloud_yaxis" | type == "halfeye_xaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")
mod_accu3_raincloud_halfeye_x <- 
    brm(scale(rating_u3) ~ dummy_raincloud_halfeye_x + effsize + (1|session), 
                          data = raincloud_vs_halfeye_x %>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_accov_raincloud_halfeye_x <- 
    brm(scale(rating_ov) ~ dummy_raincloud_halfeye_x + effsize + (1|session), 
                          data = raincloud_vs_halfeye_x %>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_diffi_raincloud_halfeye_x <- 
    brm(scale(diffi) ~ dummy_raincloud_halfeye_x + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Trying to compile a simple C file
mod_infor_raincloud_halfeye_x <- 
    brm(scale(infor) ~ dummy_raincloud_halfeye_x +
        effsize + (1|session), 
                          data = raincloud_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Trying to compile a simple C file
mod_value_raincloud_halfeye_x <- 
    brm(scale(value) ~ dummy_raincloud_halfeye_x + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_x,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Trying to compile a simple C file
mod_effic01_raincloud_halfeye_x <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_raincloud_halfeye_x + 
            effsize + (1|session),
    data = raincloud_vs_halfeye_x_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_effic0306_raincloud_halfeye_x <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_raincloud_halfeye_x +
            effsize + (1|session),
    data = raincloud_vs_halfeye_x_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
##  Raincloud Plot vs.Halfeye yaxis
raincloud_vs_halfeye_y <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_raincloud_halfeye_y = 
               case_when(type == "raincloud_yaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "halfeye_yaxis" | type == "raincloud_yaxis") %>%
    ungroup()

raincloud_vs_halfeye_y_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_raincloud_halfeye_y = 
               case_when(type == "raincloud_yaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "raincloud_yaxis" | type == "halfeye_yaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")
mod_accu3_raincloud_halfeye_y <- 
    brm(scale(rating_u3) ~ dummy_raincloud_halfeye_y + effsize + (1|session), 
                          data = raincloud_vs_halfeye_y %>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_accov_raincloud_halfeye_y <- 
    brm(scale(rating_ov) ~ dummy_raincloud_halfeye_y + effsize + (1|session), 
                          data = raincloud_vs_halfeye_y %>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_diffi_raincloud_halfeye_y <- 
    brm(scale(diffi) ~ dummy_raincloud_halfeye_y + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Trying to compile a simple C file
mod_infor_raincloud_halfeye_y <- 
    brm(scale(infor) ~ dummy_raincloud_halfeye_y + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Trying to compile a simple C file
mod_value_raincloud_halfeye_y <- 
    brm(scale(value) ~ dummy_raincloud_halfeye_y + 
            effsize + (1|session), 
                          data = raincloud_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Trying to compile a simple C file
mod_effic01_raincloud_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_raincloud_halfeye_y +
            effsize + (1|session),
    data = raincloud_vs_halfeye_y_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_effic0306_raincloud_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_raincloud_halfeye_y +
            effsize + (1|session),
    data = raincloud_vs_halfeye_y_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
sjPlot::tab_model(mod_accu3_gardneraltman_halfeye_x, 
        mod_accov_gardneraltman_halfeye_x, mod_diffi_gardneraltman_halfeye_x, 
        mod_infor_gardneraltman_halfeye_x, mod_value_gardneraltman_halfeye_x,
        mod_effic01_gardneraltman_halfeye_x, mod_effic0306_gardneraltman_halfeye_x,
        mod_accu3_gardneraltman_halfeye_y, mod_accov_gardneraltman_halfeye_y, 
        mod_diffi_gardneraltman_halfeye_y, mod_infor_gardneraltman_halfeye_y, 
        mod_value_gardneraltman_halfeye_y,
        mod_effic01_gardneraltman_halfeye_y, mod_effic0306_gardneraltman_halfeye_y,
        mod_accov_raincloud_halfeye_x, mod_diffi_raincloud_halfeye_x, 
        mod_infor_raincloud_halfeye_x, mod_value_raincloud_halfeye_x,
        mod_effic01_raincloud_halfeye_x, mod_effic0306_raincloud_halfeye_x,
        mod_accu3_raincloud_halfeye_y, mod_accov_raincloud_halfeye_y, 
        mod_diffi_raincloud_halfeye_y, mod_infor_raincloud_halfeye_y, 
        mod_value_raincloud_halfeye_y,
        mod_effic01_raincloud_halfeye_y, mod_effic0306_raincloud_halfeye_y)
  scale(rating_u3) scale(rating_ov) scale(diffi) scale(infor) scale(value) scale(log_effic_05righttrunc) scale(log_effic_05righttrunc) scale(rating_u3) scale(rating_ov) scale(diffi) scale(infor) scale(value) scale(log_effic_05righttrunc) log_effic_05righttrunc scale(rating_ov) scale(diffi) scale(infor) scale(value) scale(log_effic_05righttrunc) scale(log_effic_05righttrunc) scale(rating_u3) scale(rating_ov) scale(diffi) scale(infor) scale(value) scale(log_effic_05righttrunc) scale(log_effic_05righttrunc)
Predictors Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%)
Intercept -0.01 -0.35 – 0.33 -0.11 -0.35 – 0.13 -0.19 -0.44 – 0.08 -0.13 -0.38 – 0.12 -0.19 -0.43 – 0.06 0.34 0.08 – 0.58 0.10 -0.12 – 0.30 -0.16 -0.50 – 0.16 0.01 -0.26 – 0.27 -0.26 -0.51 – -0.03 -0.21 -0.42 – 0.02 -0.20 -0.45 – 0.05 0.42 0.16 – 0.67 9.51 9.39 – 9.64 -0.11 -0.34 – 0.11 -0.12 -0.37 – 0.11 -0.05 -0.27 – 0.18 -0.09 -0.32 – 0.16 -0.04 -0.31 – 0.25 0.07 -0.16 – 0.29 -0.16 -0.46 – 0.18 0.04 -0.20 – 0.27 -0.22 -0.46 – 0.01 -0.13 -0.34 – 0.10 -0.12 -0.36 – 0.12 0.05 -0.24 – 0.33 0.14 -0.09 – 0.38
dummy_gardneraltman_halfeye_x 0.04 -0.27 – 0.36 0.18 -0.11 – 0.46 0.37 0.26 – 0.49 0.26 0.14 – 0.39 0.35 0.23 – 0.46 -0.69 -0.75 – -0.64 -0.20 -0.24 – -0.16
effsize -0.73 -1.00 – -0.45 0.05 -0.22 – 0.31 -0.04 -0.15 – 0.06 -0.02 -0.13 – 0.09 -0.05 -0.16 – 0.05 0.00 -0.05 – 0.05 -0.00 -0.04 – 0.04 -0.50 -0.82 – -0.18 -0.08 -0.32 – 0.18 -0.04 -0.14 – 0.07 -0.03 -0.15 – 0.09 -0.01 -0.13 – 0.10 0.00 -0.04 – 0.04 0.00 -0.02 – 0.02 -0.07 -0.33 – 0.17 -0.07 -0.19 – 0.05 -0.06 -0.19 – 0.07 -0.05 -0.16 – 0.06 -0.00 -0.05 – 0.05 0.00 -0.04 – 0.04 -0.54 -0.84 – -0.22 -0.15 -0.41 – 0.10 -0.07 -0.18 – 0.05 -0.06 -0.19 – 0.06 -0.01 -0.13 – 0.11 0.00 -0.04 – 0.04 0.00 -0.03 – 0.03
dummy_gardneraltman_halfeye_y 0.29 -0.06 – 0.64 -0.04 -0.32 – 0.24 0.54 0.41 – 0.66 0.40 0.27 – 0.54 0.39 0.27 – 0.52 -0.79 -0.83 – -0.74 -0.21 -0.23 – -0.18
dummy_raincloud_halfeye_x 0.21 -0.08 – 0.50 0.25 0.12 – 0.39 0.10 -0.03 – 0.24 0.19 0.07 – 0.31 0.15 0.10 – 0.20 -0.12 -0.16 – -0.08
dummy_raincloud_halfeye_y 0.35 -0.02 – 0.73 -0.05 -0.34 – 0.23 0.43 0.30 – 0.57 0.26 0.12 – 0.39 0.24 0.11 – 0.36 -0.00 -0.05 – 0.05 -0.23 -0.27 – -0.19
Random Effects
σ2 0.64 0.95 0.43 0.46 0.40 0.35 0.58 0.78 0.83 0.46 0.55 0.47 0.25 0.23 0.98 0.55 0.62 0.50 0.31 0.54 0.78 0.92 0.54 0.60 0.54 0.26 0.51
τ00 0.27 session 0.07 session 0.59 session 0.57 session 0.62 session 0.60 session 0.43 session 0.20 session 0.22 session 0.51 session 0.45 session 0.53 session 0.69 session 0.17 session 0.04 session 0.48 session 0.41 session 0.54 session 0.79 session 0.50 session 0.15 session 0.10 session 0.46 session 0.42 session 0.49 session 0.87 session 0.53 session
ICC 0.29 0.07 0.58 0.55 0.61 0.63 0.43 0.20 0.21 0.52 0.45 0.53 0.73 0.42 0.04 0.47 0.40 0.52 0.72 0.48 0.16 0.10 0.46 0.41 0.48 0.77 0.51
N 18 session 31 session 40 session 40 session 40 session 39 session 40 session 18 session 31 session 40 session 40 session 40 session 40 session 40 session 31 session 40 session 40 session 40 session 39 session 40 session 18 session 31 session 40 session 40 session 40 session 40 session 40 session
Observations 109 185 480 480 480 1824 5472 104 177 480 480 480 1824 5472 195 480 480 480 1824 5472 107 187 480 480 480 1824 5472
Marginal R2 / Conditional R2 0.169 / 0.393 0.014 / 0.081 0.036 / 0.579 0.019 / 0.543 0.032 / 0.608 0.118 / 0.654 0.010 / 0.418 0.107 / 0.267 0.008 / 0.203 0.074 / 0.540 0.042 / 0.452 0.039 / 0.533 0.151 / 0.752 0.027 / 0.419 0.018 / 0.055 0.019 / 0.460 0.005 / 0.385 0.011 / 0.508 0.006 / 0.688 0.004 / 0.461 0.132 / 0.264 0.013 / 0.113 0.050 / 0.468 0.019 / 0.407 0.015 / 0.471 0.000 / 0.744 0.013 / 0.488
## Gardner Altman Plot vs. Raincloud Plot
gardneraltman_vs_raincloud <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_raincloud = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "raincloud_yaxis") %>%
    ungroup()

gardneraltman_vs_raincloud_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_gardneraltman_raincloud = 
               case_when(type == "gardneraltman_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "gardneraltman_xaxis" | type == "raincloud_yaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")
mod_accu3_gardneraltman_raincloud <- 
    brm(scale(rating_u3) ~ dummy_gardneraltman_raincloud + effsize + (1|session), 
                          data = gardneraltman_vs_raincloud%>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_accov_gardneraltman_raincloud <- 
    brm(scale(rating_ov) ~ dummy_gardneraltman_raincloud + effsize + (1|session), 
                          data = gardneraltman_vs_raincloud%>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_diffi_gardneraltman_raincloud <- 
    brm(scale(diffi) ~ dummy_gardneraltman_raincloud + 
            effsize + (1|session), 
                          data = gardneraltman_vs_raincloud,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Trying to compile a simple C file
mod_infor_gardneraltman_raincloud <- 
    brm(scale(infor) ~ dummy_gardneraltman_raincloud +
            effsize + (1|session), 
        data = gardneraltman_vs_raincloud,
        save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Trying to compile a simple C file
mod_value_gardneraltman_raincloud <- 
    brm(scale(value) ~ dummy_gardneraltman_raincloud + 
            effsize + (1|session), 
                          data = gardneraltman_vs_raincloud,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Trying to compile a simple C file
mod_effic01_gardneraltman_raincloud <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_raincloud + 
            effsize + (1|session),
    data = gardneraltman_vs_raincloud_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_effic0306_gardneraltman_raincloud <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_gardneraltman_raincloud + 
            effsize + (1|session),
    data = gardneraltman_vs_raincloud_effic %>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
## Halfeye xaxis vs. Halfeye yaxis

halfeye_x_vs_halfeye_y <- study1_w %>%
    group_by(session) %>%
    mutate(dummy_halfeye_x_halfeye_y = 
               case_when(type == "halfeye_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "halfeye_xaxis" | type == "halfeye_yaxis") %>%
    ungroup()

halfeye_x_vs_halfeye_y_effic <- study1_w_timestamp %>%
    group_by(session) %>%
    mutate(dummy_halfeye_x_halfeye_y = 
               case_when(type == "halfeye_xaxis" ~ 0, TRUE ~ 1)) %>%
    filter(type == "halfeye_xaxis" | type == "halfeye_yaxis") %>%
    ungroup() %>% 
    full_join(study1_w %>% select(session, effsize), by = "session")
mod_accu3_halfeye_x_halfeye_y <- 
    brm(scale(rating_u3) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y %>% 
                              filter(rating_u3_missconcept == F),
                          save_pars = save_pars(all = TRUE),
        
    silent = 2,
    refresh = 0,
    cores = 4)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_accov_halfeye_x_halfeye_y <- 
    brm(scale(rating_ov) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y%>% 
                              filter(rating_ov_missconcept == F),
                          save_pars = save_pars(all = TRUE),
        
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 20000)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
Warning: There were 1 divergent transitions after warmup. See
https://mc-stan.org/misc/warnings.html#divergent-transitions-after-warmup
to find out why this is a problem and how to eliminate them.
Warning: Examine the pairs() plot to diagnose sampling problems
mod_diffi_halfeye_x_halfeye_y <- 
    brm(scale(diffi) ~ dummy_halfeye_x_halfeye_y +
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y,
                          iter = 10000,
                          save_pars = save_pars(all = TRUE))
Compiling Stan program...
Trying to compile a simple C file
Start sampling
mod_infor_halfeye_x_halfeye_y <- 
    brm(scale(infor) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4)
Trying to compile a simple C file
mod_value_halfeye_x_halfeye_y <- 
    brm(scale(value) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session), 
                          data = halfeye_x_vs_halfeye_y,
                          save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Trying to compile a simple C file
mod_effic01_halfeye_x_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session),
    data = halfeye_x_vs_halfeye_y_effic %>%  
    filter(plotNrWithin == 1),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    iter = 10000)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
mod_effic0306_halfeye_x_halfeye_y <- 
    brm(scale(log_effic_05righttrunc) ~ dummy_halfeye_x_halfeye_y + 
            effsize + (1|session),
    data = halfeye_x_vs_halfeye_y_effic%>% 
        filter(plotNrWithin >= 4),
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    cores = 4,
    iter = 10000)
Warning: Rows containing NAs were excluded from the model.
Trying to compile a simple C file
sjPlot::tab_model(mod_accu3_halfeye_x_halfeye_y, mod_accov_halfeye_x_halfeye_y, 
                  mod_diffi_halfeye_x_halfeye_y, mod_infor_halfeye_x_halfeye_y, 
                  mod_value_halfeye_x_halfeye_y,
                  mod_effic01_halfeye_x_halfeye_y)
  scale(rating_u3) scale(rating_ov) scale(diffi) scale(infor) scale(value) scale(log_effic_05righttrunc)
Predictors Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%)
Intercept -0.10 -0.42 – 0.23 0.11 -0.10 – 0.32 -0.10 -0.34 – 0.16 -0.09 -0.33 – 0.15 -0.03 -0.29 – 0.24 0.10 -0.16 – 0.38
dummy_halfeye_x_halfeye_y 0.17 -0.15 – 0.49 -0.23 -0.51 – 0.05 0.20 0.08 – 0.33 0.16 0.03 – 0.29 0.05 -0.06 – 0.16 -0.14 -0.20 – -0.08
effsize -0.67 -0.98 – -0.38 -0.04 -0.31 – 0.22 -0.01 -0.12 – 0.10 0.04 -0.07 – 0.14 0.05 -0.05 – 0.16 -0.00 -0.05 – 0.05
Random Effects
σ2 0.70 0.98 0.48 0.47 0.41 0.37
τ00 0.25 session 0.03 session 0.56 session 0.57 session 0.65 session 0.72 session
ICC 0.26 0.03 0.53 0.55 0.62 0.66
N 18 session 31 session 40 session 40 session 40 session 39 session
Observations 109 194 480 480 480 1824
Marginal R2 / Conditional R2 0.150 / 0.337 0.019 / 0.052 0.011 / 0.521 0.008 / 0.537 0.002 / 0.598 0.005 / 0.626
sjPlot::tab_model(mod_effic0306_halfeye_x_halfeye_y,
                  mod_accu3_gardneraltman_raincloud, 
                  mod_accov_gardneraltman_raincloud, 
                  mod_diffi_gardneraltman_raincloud)
  scale(log_effic_05righttrunc) scale(rating_u3) scale(rating_ov) scale(diffi)
Predictors Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%)
Intercept 0.08 -0.15 – 0.31 0.12 -0.21 – 0.45 0.02 -0.23 – 0.27 -0.07 -0.31 – 0.16
dummy_halfeye_x_halfeye_y -0.12 -0.16 – -0.09
effsize 0.00 -0.03 – 0.03 -0.69 -0.97 – -0.39 -0.10 -0.36 – 0.15 -0.09 -0.21 – 0.03
dummy_gardneraltman_raincloud -0.17 -0.49 – 0.15 -0.01 -0.30 – 0.27 0.14 0.01 – 0.27
Random Effects
σ2 0.52 0.68 0.89 0.59
τ00 0.53 session 0.24 session 0.15 session 0.45 session
ICC 0.51 0.26 0.14 0.44
N 40 session 18 session 31 session 40 session
Observations 5472 107 178 480
Marginal R2 / Conditional R2 0.004 / 0.482 0.161 / 0.354 0.010 / 0.148 0.009 / 0.420
sjPlot::tab_model(mod_infor_gardneraltman_raincloud, 
                  mod_value_gardneraltman_raincloud,
                  mod_effic01_gardneraltman_raincloud, 
                  mod_effic0306_gardneraltman_raincloud)
  scale(infor) scale(value) scale(log_effic_05righttrunc) scale(log_effic_05righttrunc)
Predictors Estimates CI (95%) Estimates CI (95%) Estimates CI (95%) Estimates CI (95%)
Intercept -0.08 -0.30 – 0.15 -0.08 -0.33 – 0.16 0.41 0.15 – 0.68 0.05 -0.17 – 0.27
dummy_gardneraltman_raincloud 0.16 0.02 – 0.29 0.15 0.02 – 0.27 -0.77 -0.81 – -0.72 -0.10 -0.14 – -0.06
effsize -0.11 -0.23 – 0.01 -0.11 -0.23 – -0.00 -0.00 -0.04 – 0.04 -0.00 -0.04 – 0.04
Random Effects
σ2 0.60 0.50 0.25 0.57
τ00 0.43 session 0.54 session 0.70 session 0.45 session
ICC 0.42 0.52 0.73 0.44
N 40 session 40 session 39 session 40 session
Observations 480 480 1824 5472
Marginal R2 / Conditional R2 0.011 / 0.406 0.010 / 0.506 0.144 / 0.747 0.003 / 0.426

References

Arslan, Ruben C., Matthias P. Walther, and Cyril S. Tata. 2020. “Formr: A Study Framework Allowing for Automated Feedback Generation and Complex Longitudinal Experience-Sampling Studies Using R.” Behavior Research Methods 52 (1): 376–87. https://doi.org/10.3758/s13428-019-01236-y.